#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Scikit-learn完整机器学习流程实战
重点：数据预处理、特征工程、模型训练、评估、调优
"""

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.datasets import load_iris, load_diabetes, make_classification
from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV
from sklearn.preprocessing import StandardScaler, LabelEncoder, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.feature_selection import SelectKBest, f_classif
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.metrics import (accuracy_score, precision_score, recall_score, 
                           f1_score, confusion_matrix, classification_report,
                           mean_squared_error, r2_score)
from sklearn.pipeline import Pipeline
import warnings
warnings.filterwarnings('ignore')

plt.rcParams['font.sans-serif'] = ['Microsoft YaHei']
plt.rcParams['axes.unicode_minus'] = False

print("🔧 Scikit-learn完整机器学习流程实战")
print("=" * 60)

# 1. 数据加载与探索
print("1. 数据加载与探索")
print("-" * 40)

# 加载鸢尾花数据集
iris = load_iris()
X, y = iris.data, iris.target
feature_names = iris.feature_names
target_names = iris.target_names

print(f"数据集信息:")
print(f"特征数量: {X.shape[1]}")
print(f"样本数量: {X.shape[0]}")
print(f"特征名称: {feature_names}")
print(f"目标类别: {target_names}")

# 创建DataFrame便于数据探索
df = pd.DataFrame(X, columns=feature_names)
df['target'] = y
df['target_name'] = df['target'].map({i: name for i, name in enumerate(target_names)})

print(f"\n数据统计信息:")
print(df.describe())

# 2. 数据预处理
print("\n2. 数据预处理")
print("-" * 40)

# 检查缺失值
print(f"缺失值统计:")
print(df.isnull().sum())

# 数据标准化
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

print(f"标准化后的数据统计:")
print(f"均值: {np.mean(X_scaled, axis=0)}")
print(f"标准差: {np.std(X_scaled, axis=0)}")

# 3. 特征工程
print("\n3. 特征工程")
print("-" * 40)

# 特征选择
selector = SelectKBest(score_func=f_classif, k=2)
X_selected = selector.fit_transform(X, y)
selected_features = selector.get_support()

print(f"特征选择结果:")
print(f"选中的特征: {[feature_names[i] for i in range(len(feature_names)) if selected_features[i]]}")
print(f"特征得分: {selector.scores_}")

# 4. 数据集划分
print("\n4. 数据集划分")
print("-" * 40)

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)

print(f"训练集大小: {X_train.shape}")
print(f"测试集大小: {X_test.shape}")
print(f"训练集类别分布: {np.bincount(y_train)}")
print(f"测试集类别分布: {np.bincount(y_test)}")

# 5. 模型训练与评估
print("\n5. 模型训练与评估")
print("-" * 40)

# 定义多个模型
models = {
    '逻辑回归': LogisticRegression(random_state=42),
    '随机森林': RandomForestClassifier(n_estimators=100, random_state=42),
    '支持向量机': SVC(random_state=42)
}

results = {}

for name, model in models.items():
    print(f"\n{name}模型:")
    
    # 训练模型
    model.fit(X_train, y_train)
    
    # 预测
    y_pred = model.predict(X_test)
    
    # 计算评估指标
    accuracy = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, average='weighted')
    recall = recall_score(y_test, y_pred, average='weighted')
    f1 = f1_score(y_test, y_pred, average='weighted')
    
    results[name] = {
        'accuracy': accuracy,
        'precision': precision,
        'recall': recall,
        'f1': f1,
        'model': model
    }
    
    print(f"准确率: {accuracy:.3f}")
    print(f"精确率: {precision:.3f}")
    print(f"召回率: {recall:.3f}")
    print(f"F1分数: {f1:.3f}")

# 6. 模型比较与可视化
print("\n6. 模型比较与可视化")
print("-" * 40)

fig = plt.figure(figsize=(15, 10))

# 子图1: 模型性能对比
plt.subplot(2, 3, 1)
metrics = ['accuracy', 'precision', 'recall', 'f1']
colors = ['skyblue', 'lightgreen', 'lightcoral', 'gold']

x = np.arange(len(models))
width = 0.2

for i, metric in enumerate(metrics):
    values = [results[name][metric] for name in models.keys()]
    plt.bar(x + i*width, values, width, label=metric, color=colors[i])

plt.xlabel('模型')
plt.ylabel('分数')
plt.title('模型性能对比')
plt.xticks(x + width*1.5, models.keys())
plt.legend()
plt.grid(True, alpha=0.3)

# 子图2: 最佳模型混淆矩阵
best_model_name = max(results.keys(), key=lambda x: results[x]['accuracy'])
best_model = results[best_model_name]['model']
y_pred_best = best_model.predict(X_test)
cm = confusion_matrix(y_test, y_pred_best)

plt.subplot(2, 3, 2)
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', 
           xticklabels=target_names, yticklabels=target_names)
plt.xlabel('预测标签')
plt.ylabel('真实标签')
plt.title(f'{best_model_name}混淆矩阵')

# 子图3: 特征重要性（随机森林）
plt.subplot(2, 3, 3)
if hasattr(models['随机森林'], 'feature_importances_'):
    importances = models['随机森林'].feature_importances_
    indices = np.argsort(importances)[::-1]
    
    plt.barh(range(len(indices)), importances[indices], color='lightblue')
    plt.yticks(range(len(indices)), [feature_names[i] for i in indices])
    plt.xlabel('特征重要性')
    plt.title('随机森林特征重要性')

# 子图4: 交叉验证
plt.subplot(2, 3, 4)
cv_scores = {}
for name, model in models.items():
    scores = cross_val_score(model, X, y, cv=5, scoring='accuracy')
    cv_scores[name] = scores
    plt.boxplot(scores, positions=[list(models.keys()).index(name)], widths=0.6)

plt.xticks(range(len(models)), models.keys())
plt.ylabel('交叉验证准确率')
plt.title('5折交叉验证结果')
plt.grid(True, alpha=0.3)

# 子图5: 学习曲线示例
plt.subplot(2, 3, 5)
train_sizes = np.linspace(0.1, 0.9, 9)
model_lr = LogisticRegression(random_state=42)

from sklearn.model_selection import learning_curve
train_sizes, train_scores, test_scores = learning_curve(
    model_lr, X, y, cv=5, train_sizes=train_sizes
)

train_scores_mean = np.mean(train_scores, axis=1)
test_scores_mean = np.mean(test_scores, axis=1)

plt.plot(train_sizes, train_scores_mean, 'o-', label='训练集准确率')
plt.plot(train_sizes, test_scores_mean, 's-', label='验证集准确率')
plt.xlabel('训练样本数')
plt.ylabel('准确率')
plt.title('学习曲线')
plt.legend()
plt.grid(True, alpha=0.3)

# 子图6: 模型预测概率（逻辑回归）
plt.subplot(2, 3, 6)
if hasattr(models['逻辑回归'], 'predict_proba'):
    y_proba = models['逻辑回归'].predict_proba(X_test)
    
    for i in range(len(target_names)):
        plt.hist(y_proba[y_test == i, i], alpha=0.7, 
                label=target_names[i], bins=10)
    
    plt.xlabel('预测概率')
    plt.ylabel('频数')
    plt.title('逻辑回归预测概率分布')
    plt.legend()

plt.tight_layout()
plt.savefig('sklearn_complete_pipeline.png', dpi=300, bbox_inches='tight')
plt.show()

# 7. 超参数调优
print("\n7. 超参数调优")
print("-" * 40)

# 随机森林参数调优
param_grid = {
    'n_estimators': [50, 100, 200],
    'max_depth': [None, 3, 5, 10],
    'min_samples_split': [2, 5, 10]
}

rf = RandomForestClassifier(random_state=42)
grid_search = GridSearchCV(rf, param_grid, cv=5, scoring='accuracy')
grid_search.fit(X_train, y_train)

print(f"最佳参数: {grid_search.best_params_}")
print(f"最佳交叉验证分数: {grid_search.best_score_:.3f}")

# 8. Pipeline使用
print("\n8. Pipeline完整流程")
print("-" * 40)

# 创建包含预处理和模型的pipeline
pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('classifier', LogisticRegression(random_state=42))
])

# 使用pipeline训练和预测
pipeline.fit(X_train, y_train)
y_pred_pipeline = pipeline.predict(X_test)
accuracy_pipeline = accuracy_score(y_test, y_pred_pipeline)

print(f"Pipeline准确率: {accuracy_pipeline:.3f}")

# 9. 实战应用指南
print("\n9. 实战应用指南")
print("-" * 40)

print("🔧 Scikit-learn核心工作流程:")
print("1. 数据加载与探索 → 理解数据特征和分布")
print("2. 数据预处理 → 处理缺失值、标准化")
print("3. 特征工程 → 特征选择、特征创造")
print("4. 模型选择 → 根据问题类型选择合适模型")
print("5. 模型训练 → 使用训练数据拟合模型")
print("6. 模型评估 → 使用测试数据评估性能")
print("7. 超参数调优 → 优化模型参数")
print("8. 模型部署 → 应用于实际问题")

print("\n💡 最佳实践:")
print("- 始终进行数据探索和可视化")
print("- 使用交叉验证评估模型稳定性")
print("- 优先使用Pipeline确保数据一致性")
print("- 关注模型的可解释性和业务价值")

# 10. 练习任务
print("\n10. 练习任务")
print("-" * 40)

print("练习1: 糖尿病数据集回归问题")
print("- 使用load_diabetes()加载数据")
print("- 实现完整的回归分析流程")
print("- 比较多种回归模型的性能")

print("\n练习2: 自定义分类数据集")
print("- 使用make_classification生成数据")
print("- 实现特征选择和模型调优")
print("- 分析不同参数对性能的影响")

print("\n练习3: 真实数据集应用")
print("- 下载Kaggle或UCI的公开数据集")
print("- 实现端到端的机器学习项目")
print("- 撰写项目报告和结果分析")

print("\n" + "=" * 60)
print("🎯 学习要点总结:")
print("1. 掌握了Scikit-learn的完整工作流程")
print("2. 理解了数据预处理和特征工程的重要性")
print("3. 学会了多种模型评估和比较方法")
print("4. 掌握了超参数调优和Pipeline使用")

print("\n下一步项目:")
print("- 泰坦尼克生存预测项目")
print("- 鸢尾花分类项目")
print("- 房价预测项目")

print("\n📚 推荐资源:")
print("- Scikit-learn官方文档")
print("- 《Python机器学习实战》")
print("- Kaggle入门竞赛")

print("\n🚀 运行建议:")
print("python Scikit-learn完整流程.py")
print("观察6个子图的可视化结果，理解完整机器学习流程")